From 9b6639ad3cd51f07a4261a5f947090496e33a252 Mon Sep 17 00:00:00 2001 From: Max Nuding Date: Tue, 11 Jan 2022 15:51:04 +0100 Subject: [PATCH] parsing test --- nebula_rss/nebula_loader.py | 12 +++++- nebula_rss/nebula_video.py | 1 + poetry.lock | 78 ++++++++++++++++++------------------- pyproject.toml | 2 +- tests/test_nebula_rss.py | 16 +++++++- 5 files changed, 66 insertions(+), 43 deletions(-) diff --git a/nebula_rss/nebula_loader.py b/nebula_rss/nebula_loader.py index 2abc5ac..4023af1 100755 --- a/nebula_rss/nebula_loader.py +++ b/nebula_rss/nebula_loader.py @@ -1,4 +1,5 @@ import datetime +import logging import os import re import time @@ -10,6 +11,7 @@ from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.firefox.options import Options from selenium.webdriver.firefox.service import Service from selenium.webdriver.support.wait import WebDriverWait +from selenium.webdriver.remote.remote_connection import LOGGER from bs4 import BeautifulSoup from nebula_rss import NebulaVideo @@ -24,12 +26,18 @@ class NebulaLoader: ): self.username = username self.password = password + + LOGGER.setLevel(logging.FATAL) service = None if driver_path: - service = Service(driver_path()) + service = Service(driver_path) options = Options() options.headless = True - self.driver = webdriver.Firefox(service=service, options=options) + self.driver = webdriver.Firefox( + service=service, + options=options, + log_path=os.devnull, + service_log_path=os.devnull) self.driver.implicitly_wait(10) # seconds @staticmethod diff --git a/nebula_rss/nebula_video.py b/nebula_rss/nebula_video.py index 4370b4c..aa3fcf7 100644 --- a/nebula_rss/nebula_video.py +++ b/nebula_rss/nebula_video.py @@ -1,6 +1,7 @@ from dataclasses import dataclass import datetime + @dataclass class NebulaVideo: """Defines a video on Nebula""" diff --git a/poetry.lock b/poetry.lock index 6bfe19d..d8a2fb2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -128,6 +128,14 @@ category = "main" optional = false python-versions = ">=3.5" +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "lxml" version = "4.7.1" @@ -142,14 +150,6 @@ html5 = ["html5lib"] htmlsoup = ["beautifulsoup4"] source = ["Cython (>=0.29.7)"] -[[package]] -name = "more-itertools" -version = "8.12.0" -description = "More routines for operating on iterables, beyond itertools" -category = "dev" -optional = false -python-versions = ">=3.5" - [[package]] name = "outcome" version = "1.1.0" @@ -174,14 +174,15 @@ pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" [[package]] name = "pluggy" -version = "0.13.1" +version = "1.0.0" description = "plugin and hook calling mechanisms for python" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.6" [package.extras] dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "py" @@ -228,24 +229,23 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "5.4.3" +version = "6.2.5" description = "pytest: simple powerful testing with Python" category = "dev" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" [package.dependencies] atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=17.4.0" +attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} -more-itertools = ">=4.0.0" +iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<1.0" -py = ">=1.5.0" -wcwidth = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" [package.extras] -checkqa-mypy = ["mypy (==v0.761)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] [[package]] @@ -304,6 +304,14 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + [[package]] name = "trio" version = "0.19.0" @@ -353,14 +361,6 @@ brotli = ["brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] -[[package]] -name = "wcwidth" -version = "0.2.5" -description = "Measures the displayed width of unicode strings in a terminal" -category = "dev" -optional = false -python-versions = "*" - [[package]] name = "wsproto" version = "1.0.0" @@ -375,7 +375,7 @@ h11 = ">=0.9.0,<1" [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "62f191b8a92cdb5ce1da2df6d85f052f1a6db3f07b9420183693ced1941cdd4f" +content-hash = "feac0aede04b3bbace7c3b2a6c9f0240e6ee1210abd4b598083952fcaa185e20" [metadata.files] async-generator = [ @@ -490,6 +490,10 @@ idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] lxml = [ {file = "lxml-4.7.1-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:d546431636edb1d6a608b348dd58cc9841b81f4116745857b6cb9f8dadb2725f"}, {file = "lxml-4.7.1-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6308062534323f0d3edb4e702a0e26a76ca9e0e23ff99be5d82750772df32a9e"}, @@ -552,10 +556,6 @@ lxml = [ {file = "lxml-4.7.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:447d5009d6b5447b2f237395d0018901dcc673f7d9f82ba26c1b9f9c3b444b60"}, {file = "lxml-4.7.1.tar.gz", hash = "sha256:a1613838aa6b89af4ba10a0f3a972836128801ed008078f8c1244e65958f1b24"}, ] -more-itertools = [ - {file = "more-itertools-8.12.0.tar.gz", hash = "sha256:7dc6ad46f05f545f900dd59e8dfb4e84a4827b97b3cfecb175ea0c7d247f6064"}, - {file = "more_itertools-8.12.0-py3-none-any.whl", hash = "sha256:43e6dd9942dffd72661a2c4ef383ad7da1e6a3e968a927ad7a6083ab410a688b"}, -] outcome = [ {file = "outcome-1.1.0-py2.py3-none-any.whl", hash = "sha256:c7dd9375cfd3c12db9801d080a3b63d4b0a261aa996c4c13152380587288d958"}, {file = "outcome-1.1.0.tar.gz", hash = "sha256:e862f01d4e626e63e8f92c38d1f8d5546d3f9cce989263c521b2e7990d186967"}, @@ -565,8 +565,8 @@ packaging = [ {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] pluggy = [ - {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, - {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, @@ -585,8 +585,8 @@ pyparsing = [ {file = "pyparsing-3.0.6.tar.gz", hash = "sha256:d9bdec0013ef1eb5a84ab39a3b3868911598afa494f5faa038647101504e2b81"}, ] pytest = [ - {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"}, - {file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"}, + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, ] python-dateutil = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, @@ -611,6 +611,10 @@ soupsieve = [ {file = "soupsieve-2.3.1-py3-none-any.whl", hash = "sha256:1a3cca2617c6b38c0343ed661b1fa5de5637f257d4fe22bd9f1338010a1efefb"}, {file = "soupsieve-2.3.1.tar.gz", hash = "sha256:b8d49b1cd4f037c7082a9683dfa1801aa2597fb11c3a1155b7a5b94829b4f1f9"}, ] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] trio = [ {file = "trio-0.19.0-py3-none-any.whl", hash = "sha256:c27c231e66336183c484fbfe080fa6cc954149366c15dc21db8b7290081ec7b8"}, {file = "trio-0.19.0.tar.gz", hash = "sha256:895e318e5ec5e8cea9f60b473b6edb95b215e82d99556a03eb2d20c5e027efe1"}, @@ -623,10 +627,6 @@ urllib3 = [ {file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"}, {file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"}, ] -wcwidth = [ - {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, - {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, -] wsproto = [ {file = "wsproto-1.0.0-py3-none-any.whl", hash = "sha256:d8345d1808dd599b5ffb352c25a367adb6157e664e140dbecba3f9bc007edb9f"}, {file = "wsproto-1.0.0.tar.gz", hash = "sha256:868776f8456997ad0d9720f7322b746bbe9193751b5b290b7f924659377c8c38"}, diff --git a/pyproject.toml b/pyproject.toml index c0c2cb7..82d42a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ lxml = "^4.7.1" bs4 = "^0.0.1" [tool.poetry.dev-dependencies] -pytest = "^5.2" +pytest = "^6.2" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/tests/test_nebula_rss.py b/tests/test_nebula_rss.py index 8fb0b2d..b6fceef 100644 --- a/tests/test_nebula_rss.py +++ b/tests/test_nebula_rss.py @@ -1,5 +1,19 @@ -from nebula_rss import __version__ +import datetime +from nebula_rss import __version__, NebulaLoader +from bs4 import BeautifulSoup def test_version(): assert __version__ == '0.1.0' + + +def test_video_parsing(): + example_div = '
Half as Interesting avatar
The Bug That Created “Free Public Wifi” Networks That Didn’t Work
Half as Interesting•Video published:
' # noqa + soup = BeautifulSoup(example_div, features='lxml') + anchor = soup.div.a + video = NebulaLoader._parse_anchor(anchor) + assert video.title == 'The Bug That Created “Free Public Wifi” Networks That Didn’t Work' + assert video.creator == 'Half as Interesting' + assert video.url == 'https://nebula.app/videos/hai-the-bug-that-created-free-public-wifi-networks-that-didnt-work' + assert video.release_at == datetime.datetime(2022, 1, 6, 15, 39, 39, tzinfo=datetime.timezone.utc) + print(video)